# coding=utf8

import re
import logging
logger = logging.getLogger('debug')

wiki_file = 'data/wiki_data.txt'
word2vec_train_file = 'data/word2vec_train_data.txt'


def connect_entity_words():
    '''
    保留实体的完整性，在多个词语之间加上下划线
    '''
    wiki_data = open(wiki_file).readlines()
    f_write = open(word2vec_train_file, 'w')
    for line in wiki_data:
        m_list = re.findall(r'====(.+?)@.+?\+\+\+\+', line)
        m_list = [m_i.replace(' ', '_') for m_i in m_list]
        m_list = [m_i.replace('\\', '') for m_i in m_list]
        for i in xrange(len(m_list)):
            line = re.sub(r'====(.+?)@.+?\+\+\+\+', m_list[i], line, 1)
        f_write.write(line)
    f_write.close()
    

def get_pure_train_data():

    def del_irrelevant_char(line):
        '''
        去除非英文字符
        '''
        word_list_res = []
        punctuation = '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'    # 借鉴了string.punctuation
        word_list = line.translate(None, punctuation).split()
        for word in word_list:
            irrelevant_char = False
            for c in word:
                if ord(c)<0 or ord(c)>128:
                    irrelevant_char = True
                    break

            if word.isdigit():
                pass
            elif irrelevant_char:
                pass
            else:
                word_list_res.append(word)
        return word_list_res

    print'进行数据预处理...'
    lines = open(word2vec_train_file).readlines()
    f_write = open(word2vec_train_file, 'w')

    lastline = ''
    prompt = 0
    progress = len(lines) / 100
    for line in lines:
        prompt += 1
        if prompt%progress == 0:
            print '当前进度是%.2f%%' %(100.0*prompt/len(lines))

        if line != lastline:
            word_list = del_irrelevant_char(line)
            for word in word_list:
                f_write.write(word+' ')
            lastline = line
